//下载器
const http = require('http');
const https = require('https');
//
const querystring = require('querystring');
//解析URL地址
const URL = require('url').URL;
const Request = require('./node-request').Request;
const Response = require('./node-response').Response;
//
var async = require('async');
var error = new Array();
/**
 *   基于http协议的下载器第二个版本 ，增加了并发控制以及失败重试设置
 *   理论上只要继承这个类，并且改写loadContent这个方法就能支持更多的可能性
 *
 */
class Grabber {


    /**
     * 爬虫的延迟时间和最大并发量
     * @param time 每一个请求的延迟时间
     * @param maxCount 最大的并发请求数量
     * @param retryCount 请求失败的重试次数
     * @param interval 重试的时间间隔
     */
    constructor({time=0,maxCount=32,retryCount=3,interval=2000}={}){

        this.queue = async.queue((task,callbacks)=>{
                //设置重试次数
                async.retry({times:retryCount,interval:interval},(tast)=>{
                    this.loadContent(task,tast);
                },(err,resp)=>{
                    if (err!=null){
                        error.push(resp);
                    }
                    callbacks(resp);
                });
;
            //
        },maxCount);
        this.queue.saturated = ()=>{console.log("任务将完成")};
        this.queue.empty = ()=>{console.log("正在执行最后任务执行完成")};
        this.queue.drain = function() {
            console.log('all tasks have been processed');
        };
        process.on('exit',()=>console.log("失败的链接:"+error));
    }

    addRxRequest(request,callbacks){

        this.queue.push(request,callbacks);

    }

    //发送请求
    loadContent(request,callback){
        console.log('当前抓取网页:'+request.url);
        let urlp = new URL(request.url);

        if(request.url.startsWith("http://")){
            //get方法
            let queryString = request.url.split('?').filter((str,index)=>index==1).toString();
            let options = {
                hostname: urlp.hostname,
                port: urlp.port,
                path: urlp.pathname+"?"+queryString,
                method: request.method,
                headers: request.headers
            };
            var result = '';
            const req = http.request(options, (res) => {
                res.setEncoding('utf8');
                res.on('data', (chunk) => {
                    result+=chunk;
                });
                res.on('end', () => {
                    // console.log(`响应头: ${JSON.stringify(res.headers["set-cookie"])}`);
                    let resp = new Response({text:result,url:request.url,headers:res.headers});
                    callback(null,resp);
                });
            });

            req.on('error', (e) => {
                console.error(`请求遇到问题: ${e.message}:${request.url}`);
                callback(e,request.url);
            });
            req.on('timeout',()=>{
               console.log("请求超时："+request.url);
               req.abort();
            });
            //post方法
            if(request.method == 'POST'){
                queryString = queryString.stringify(request.parameter);
                req.write(queryString);
            }
            req.end();

        }
        else if(request.url.startsWith("https://")){
            let queryString = request.url.split('?').filter((str,index)=>index==1).toString();
            let options = {
                hostname: urlp.hostname,
                port: 443,
                path: urlp.pathname+"?"+queryString,
                method: request.method,
                headers: request.headers
            };

            var result = '';
            const req = https.request(options, (res) => {
                res.setEncoding('utf8');

                res.on('data', (chunk) => {
                    result+=chunk;
                });
                res.on('end', () => {
                    let resp = new Response({text:result,url:request.url,headers:res.headers});
                    callback(null,resp);
                });
            });

            req.on('error', (e) => {
                console.error(`请求遇到问题: ${e.message}`);
                callback(e,request.url);
            });
            req.on('timeout',()=>{req.abort()});
            //如果为post请求
            if(request.method == 'POST'){
                queryString = queryString.stringify(request.parameter);
                req.write(queryString);
            }
            req.end();
        }
        else{
            throw new Error("url地址错误");
        }
    }


}

// function loadHtml(request,callback){
//     console.log('当前抓取网页:'+request.url);
//     let urlp = new URL(request.url);
//
//     if(request.url.startsWith("http://")){
//         //get方法
//         let queryString = request.url.split('?').filter((str,index)=>index==1).toString();
//         let options = {
//             hostname: urlp.hostname,
//             port: urlp.port,
//             path: urlp.pathname+"?"+queryString,
//             method: request.method,
//             headers: request.headers
//         };
//         var result = '';
//         const req = http.request(options, (res) => {
//             res.setEncoding('utf8');
//             res.on('data', (chunk) => {
//                 result+=chunk;
//             });
//             res.on('end', () => {
//                 // console.log(`响应头: ${JSON.stringify(res.headers["set-cookie"])}`);
//                 let resp = new Response({text:result,url:request.url,headers:res.headers});
//                 callback(resp);
//             });
//         });
//
//         req.on('error', (e) => {
//             console.error(`请求遇到问题: ${e.message}`);
//         });
//         //post方法
//         if(request.method == 'POST'){
//             queryString = queryString.stringify(request.parameter);
//             req.write(queryString);
//         }
//         req.end();
//
//     }
//     else if(request.url.startsWith("https://")){
//         let queryString = request.url.split('?').filter((str,index)=>index==1).toString();
//         let options = {
//             hostname: urlp.hostname,
//             port: 443,
//             path: urlp.pathname+"?"+queryString,
//             method: request.method,
//             headers: request.headers
//         };
//
//         var result = '';
//         const req = https.request(options, (res) => {
//             res.setEncoding('utf8');
//
//             res.on('data', (chunk) => {
//                 result+=chunk;
//             });
//             res.on('end', () => {
//                 let resp = new Response({text:result,url:request.url,headers:res.headers});
//                 callback(resp);
//             });
//         });
//
//         req.on('error', (e) => {
//             console.error(`请求遇到问题: ${e.message}`);
//         });
//         //如果为post请求
//         if(request.method == 'POST'){
//             queryString = queryString.stringify(request.parameter);
//             req.write(queryString);
//         }
//         req.end();
//     }
//     else{
//         throw new Error("url地址错误");
//     }
// }
module.exports.Grabber = Grabber;
